import scrapy
from scrapy import FormRequest,Request
from scrapy.selector import Selector
import sys
import time
from tutorial.items import SMTXItem
reload(sys)
sys.setdefaultencoding('utf8')

class SMTX2Spider(scrapy.Spider):
    name = "miaotianxia2"
    custom_settings = {'ITEM_PIPELINES':{'tutorial.pipelines.SMTXPipeline':400,}}
    allowed_domains = ["miaotianxia.com"]
    start_urls = ['https://user.miaotianxia.com/index']
    loginokurl = 'https://user.miaotianxia.com/index'
    login_url = 'https://user.miaotianxia.com/public/login'
    productinfolist =[]
    GetthisAll = True
    today=  time.strftime("%Y-%m-%d", time.localtime())

    if GetthisAll:
        for num in range(1,2):
            start_urls.append('https://www.miaotianxia.com/product/buy?p='+str(num))
    else:
        for num in range(1,1):
            start_urls.append('https://www.miaotianxia.com/product/buy?p='+str(num))

    def parse(self, response):
        #print str(response.text).encode('utf-8')
        #response.text.encode('gb18030')
        strdata =str( response.text.encode('utf-8'))
        #print strdata
        sel = Selector(text=strdata)
        #table = sel.xpath('/html/body/div[4]/div[2]/div[2]/div/table')
        #print table
        trs = sel.xpath('//table[@class="tablelist"]/tr')
        #print trs
        for tr in trs:
            productaddress ='https://www.miaotianxia.com'+ str(tr.xpath('td[2]/a/@href').extract()[0].encode('utf-8')).replace("(","").replace(")","")
            yield scrapy.Request(productaddress,callback=self.GetDetailInfo)

    def GetDetailInfo(self,response):
        strdata =str( response.text.encode('utf-8'))
        sel = Selector(text=strdata)
        items = []
        item = SMTXItem()
        img = 'https://www.miaotianxia.com'+sel.xpath('//*[@id="productalbum"]/div[1]/img/@src').extract()[0]
        title = sel.xpath('//*[@id="productinfo"]/h2/text()').extract()[0]

        requireDesPS =sel.xpath('//*[@id="productinfo"]/ul[2]')
        desc = requireDesPS.xpath('string(.)').extract()[0]


        link = response.url
        people = sel.xpath('//*[@id="encard"]/ul/li[1]/text()').extract()[0]
        tel = sel.xpath('//*[@id="encard"]/ul/li[2]/a/text()').extract()[0]
        email = sel.xpath('//*[@id="encard"]/ul/li[3]/a/text()').extract()[0]
        pubTime = self.today
        num = sel.xpath('//*[@id="productinfo"]/ul[1]/li[1]/text()').extract()[0]

        item['title'] = title
        item['link'] =  link
        item['desc'] = str(desc).strip()
        item['tel'] =   tel
        item['people'] = people
        item['pubTime'] = pubTime
        item['num']=num
        item['email']=email
        item['img']=img
        items.append(item)
        return item


    def start_requests(self):
        return [scrapy.FormRequest("https://user.miaotianxia.com/public/login",
                               formdata={'username':'fakeyoulife','password':'fakeyoulife','referer':""},
                               callback=self.after_login)]

    def after_login(self,response):
        #yield scrapy.Request(self.loginokurl,callback=self.parse)
        for url in self.start_urls:
            yield scrapy.Request(url)
